{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ef92c15d",
   "metadata": {},
   "source": [
    "## Metrics\n",
    "\n",
    "汇总常见2分类的指标，例如: AUC，ROC曲线，ACC, 敏感性， 特异性，精确度，召回率，PPV, NPV, F1\n",
    "\n",
    "具体的介绍，可以参考一下：https://blog.csdn.net/sunflower_sara/article/details/81214897"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "58a846b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Acc</th>\n",
       "      <th>AUC</th>\n",
       "      <th>95% CI</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>PPV</th>\n",
       "      <th>NPV</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.892317</td>\n",
       "      <td>0.965521</td>\n",
       "      <td>0.9585-0.9725</td>\n",
       "      <td>0.888518</td>\n",
       "      <td>0.929032</td>\n",
       "      <td>0.991803</td>\n",
       "      <td>0.463023</td>\n",
       "      <td>0.991803</td>\n",
       "      <td>0.888518</td>\n",
       "      <td>0.937324</td>\n",
       "      <td>0.896</td>\n",
       "      <td>label=0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.886267</td>\n",
       "      <td>0.951541</td>\n",
       "      <td>0.9254-0.9777</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>0.885740</td>\n",
       "      <td>0.057789</td>\n",
       "      <td>0.999656</td>\n",
       "      <td>0.057789</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>0.109005</td>\n",
       "      <td>0.217</td>\n",
       "      <td>label=1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.900484</td>\n",
       "      <td>0.977007</td>\n",
       "      <td>0.9698-0.9843</td>\n",
       "      <td>0.978022</td>\n",
       "      <td>0.898289</td>\n",
       "      <td>0.213942</td>\n",
       "      <td>0.999308</td>\n",
       "      <td>0.213942</td>\n",
       "      <td>0.978022</td>\n",
       "      <td>0.351085</td>\n",
       "      <td>0.223</td>\n",
       "      <td>label=3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.881428</td>\n",
       "      <td>0.953164</td>\n",
       "      <td>0.9387-0.9676</td>\n",
       "      <td>0.924242</td>\n",
       "      <td>0.880556</td>\n",
       "      <td>0.136161</td>\n",
       "      <td>0.998251</td>\n",
       "      <td>0.136161</td>\n",
       "      <td>0.924242</td>\n",
       "      <td>0.237354</td>\n",
       "      <td>0.188</td>\n",
       "      <td>label=5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.842710</td>\n",
       "      <td>0.948169</td>\n",
       "      <td>0.9355-0.9609</td>\n",
       "      <td>0.942308</td>\n",
       "      <td>0.839475</td>\n",
       "      <td>0.160131</td>\n",
       "      <td>0.997773</td>\n",
       "      <td>0.160131</td>\n",
       "      <td>0.942308</td>\n",
       "      <td>0.273743</td>\n",
       "      <td>0.105</td>\n",
       "      <td>label=6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Acc       AUC         95% CI  Sensitivity  Specificity       PPV  \\\n",
       "0  0.892317  0.965521  0.9585-0.9725     0.888518     0.929032  0.991803   \n",
       "1  0.886267  0.951541  0.9254-0.9777     0.958333     0.885740  0.057789   \n",
       "2  0.900484  0.977007  0.9698-0.9843     0.978022     0.898289  0.213942   \n",
       "3  0.881428  0.953164  0.9387-0.9676     0.924242     0.880556  0.136161   \n",
       "4  0.842710  0.948169  0.9355-0.9609     0.942308     0.839475  0.160131   \n",
       "\n",
       "        NPV  Precision    Recall        F1  Threshold    Label  \n",
       "0  0.463023   0.991803  0.888518  0.937324      0.896  label=0  \n",
       "1  0.999656   0.057789  0.958333  0.109005      0.217  label=1  \n",
       "2  0.999308   0.213942  0.978022  0.351085      0.223  label=3  \n",
       "3  0.998251   0.136161  0.924242  0.237354      0.188  label=5  \n",
       "4  0.997773   0.160131  0.942308  0.273743      0.105  label=6  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy  as np\n",
    "from onekey_algo.custom.components import metrics\n",
    "\n",
    "# log_path 修改为Onekey val目录中对应的log文件。\n",
    "log_path = r'C:/Users/onekey/Desktop/onekey_comp/comp4-What（分类识别）/20220626/resnet18/train/Epoch-2.txt'\n",
    "val_log = pd.read_csv(log_path, names=['fname', 'pred_score', 'pred_label', 'gt'], sep='\\t')\n",
    "ul_labels = np.unique(val_log['pred_label'])\n",
    "\n",
    "metric_results = []\n",
    "for ul in ul_labels:\n",
    "    pred_score = list(map(lambda x: x[0] if x[1] == ul else 1-x[0], np.array(val_log[['pred_score', 'pred_label']])))\n",
    "    gt = [1 if gt_ == ul else 0 for gt_ in np.array(val_log['gt'])]\n",
    "    acc, auc, ci, tpr, tnr, ppv, npv, precision, recall, f1, thres = metrics.analysis_pred_binary(gt, pred_score)\n",
    "    ci = f\"{ci[0]:.4f}-{ci[1]:.4f}\"\n",
    "    metric_results.append([acc, auc, ci, tpr, tnr, ppv, npv, precision, recall, f1, thres, f'label={ul}'])\n",
    "\n",
    "pd.DataFrame(metric_results, \n",
    "             columns=['Acc', 'AUC', '95% CI', 'Sensitivity', 'Specificity', 'PPV', 'NPV', \n",
    "                      'Precision', 'Recall', 'F1', 'Threshold', 'Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "047b609e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
